package com.mycrawler;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

public class FilePinelinePageProcessor implements PageProcessor {


    @Override
    public void process(Page page) {
        Html html = page.getHtml();

        Selectable links = html.links();


        List<String> allLinks = links.all();
//        把链接地址添加到访问队列中
        page.addTargetRequests(allLinks);
//      把页面传递给pipeline 由pipeline保存到磁盘
        page.putField("html",html.get());
    }

    @Override
    public Site getSite() {
        return Site.me();
    }


    public static void main(String[] args) {
        FilePipeline filePipeline = new FilePipeline();
        filePipeline.setPath("d://html");

//      创建一个scheduler
        QueueScheduler queueScheduler =  new QueueScheduler();
//      指定队列使用布隆过滤器去重
        queueScheduler.setDuplicateRemover(new BloomFilterDuplicateRemover(10000000));

        Spider.create(new FilePinelinePageProcessor()).addUrl("http://itheima.com")
                .setScheduler(queueScheduler).addPipeline(new ConsolePipeline()).addPipeline(filePipeline).start();
    }
}
